#-*- encoding:utf-8 -*-
from time import time
from sklearn.datasets import load_files
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from matplotlib import pyplot as plt

#加载数据
print('loading train dataset...')
t = time()
news_train = load_files(r'20news-bydate\20news-bydate\20news-bydate-train')
print('\nsummary:{0} documents in {1} categories.'.format(len(news_train.data),len(news_train.target_names)))
print('\ndone in {0} seconds'.format(time() - t))

t = time()
print('\nvectorizing train dataset...')
vectorizer = TfidfVectorizer(encoding='latin-1')
X_train = vectorizer.fit_transform((d for d in news_train.data))
print('\nn_sample:%d , n_features: %d' % X_train.shape)
print('\nnumber of non_zero features in shape [{0}]:{1}'.
      format(news_train.filenames[0],X_train[0].getnnz()))
print('\ndown in {0} seconds'.format(time()-t))

#模型训练
print ('\ntraining models ..'.format(time() - t))
t = time()
y_train = news_train.target
clf = MultinomialNB(alpha=0.001)#alpha为平滑指数
clf.fit(X_train,y_train)
train_score = clf.score(X_train,y_train)
print('\ntrain score:{0}'.format(train_score))
print('\ndone in {0} secondes'.format(time() - t))

print('\nloading test datasets ...')
news_test = load_files(r'20news-bydate\20news-bydate\20news-bydate-test')
#将文本向量化
print('\nvectorizing test dataset ...')
X_test = vectorizer.transform((d for d in news_test.data))
y_test = news_test.target
print('\npredicting test tataset...')
pred = clf.predict(X_test)
print('\npredict score:{}'.format(clf.score(X_test,y_test)))

#查看预测准确性
print('\nclassification report on test set for classifier:')
print(clf)
print(classification_report(y_test,pred,target_names = news_train.target_names))

#生成混淆矩阵
cm = confusion_matrix(y_test,pred)
print('\nconfusion_mtrix:')
print(cm)

#对混淆矩阵可视化
plt.figure(figsize=(8,8),dpi = 60)
plt.title('Confusion matrix of the Classifier')
ax = plt.gca()
ax.spines['right'].set_color('none')            
ax.spines['top'].set_color('none')
ax.spines['bottom'].set_color('none')
ax.spines['left'].set_color('none')
ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')
ax.set_xticklabels([])
ax.set_yticklabels([])
plt.matshow(cm, fignum=1, cmap='gray')
plt.colorbar()











